﻿using System;
using System.Collections.Generic;
using System.Text;
using NSoup.Select;
using Com.Toook.AutoDownloadHtml.DownloadHtml.ParsePage.Entity;
using Com.Toook.AutoDownloadHtml.DownloadHtml.ParsePage.Enum;
using NSoup.Nodes;
using Com.Toook.AutoDownloadHtml.Util;
using System.Collections.Specialized;
using System.Text.RegularExpressions;

namespace Com.Toook.AutoDownloadHtml.DownloadHtml.ParsePage
{
    /// <summary>
    /// 根据Step解析出分类页面中详细页面的URL(分类页面url固定，只有一个)
    /// </summary>
    public class GetKindResultFromHtml
    {
        private NSoup.Nodes.Document doc = null;
        private string xmlFilePath;
        private Kind kind;
        /// <summary>
        /// 保存分类页面中解析出的所有详细页面Url，防止添加重复的Url
        /// </summary>
        private StringDictionary sdUrlResult = new StringDictionary();
        private int minLinkTextLength = 0;
        /// <summary>
        /// 1、获取分类中的详细页面URL，下一页Url等内容,保存到数据库
        /// 2、读取数据库中Url等，再解析出内容保存到数据库
        /// </summary>
        /// <param name="xmlFilePath">配置文件路径</param>
        /// <param name="siteKindId">站点分类ID</param>
        public GetKindResultFromHtml(string xmlFilePath, string siteKindId)
        {
            this.xmlFilePath = xmlFilePath;
            CreateKindByXml createKind = new CreateKindByXml(xmlFilePath);
            this.kind = createKind.GetKind(siteKindId);
        }
        
        /// <summary>
        /// 记录页面Url全部存在的页数。连续2页的页面记录数全部存在，跳出此分类的抓取。
        /// </summary>
        private int pageStopCrawlCount = 0;
        /// <summary>
        /// 记录当前分类页面url
        /// </summary>
        private string currentPageUrl = "";
        /// <summary>
        /// 保存页面的初始根元素
        /// </summary>
        private Elements rootElements = new Elements();
        /// <summary>
        /// 标识分类页面是否有新的Url等。如果有抓完列表后，需要继续抓取内容。
        /// </summary>
        private bool hasNewUrlList = false;
        /// <summary>
        /// 设置页面初始根元素
        /// </summary>
        /// <param name="url"></param>
        /// <param name="step"></param>
        private void InitRootElements(string url, Step step)
        {
            this.rootElements = GetInitRootElements(url, step);
        }
        /// <summary>
        /// 自动解析出分类页面结果
        /// </summary>
        /// <param name="initElements"></param>
        public void GetKindPageResults()
        {
            this.minLinkTextLength = this.kind.MinTitleLength;

            this.currentPageUrl = this.kind.BaseUrl;
            InitRootElements(currentPageUrl, this.kind.InitStep);
            //获取第一页数据
            RunPages();               

            //如果有分页规则，解析后面的分页
            if (kind.PageRule != null)
            {
                while (true)
                {
                    string nextPageUrl = GetNextPageUrl();//从当前页面中获取下一页的地址
                    //如果 1、没有下一页 2、连续2页的记录都存在 3、下一页与当前页相同。 停止此分类抓取
                    if (string.IsNullOrEmpty(nextPageUrl) || pageStopCrawlCount == 2 || this.currentPageUrl == nextPageUrl)
                    {
                        break;
                    }
                    InitRootElements(nextPageUrl, kind.InitStep);//将rootElements设置为下一页的初始元素
                    this.currentPageUrl = nextPageUrl;
                    RunPages();//继续对下一页进行解析 
                }                
            }
            //没有分页的，完成此分类页面的解析。 
            //然后判断是否有新增Url，有就继续解析内容。
            if (hasNewUrlList)
            {
                GetContent();//根据数据库表中的url,获取内容，保存最终内容结果
            }
            //GetContent();
        }
        /// <summary>
        /// 根据数据库表中的url,获取内容
        /// </summary>
        private void GetContent()
        {
            CreateContentByXml createContent = new CreateContentByXml(this.xmlFilePath);
            DetailRoadMap roadMap = createContent.GetDetailRoadMap(this.kind.SiteKindId);
            GetContentResultFromHtml html = new GetContentResultFromHtml(roadMap);
            html.GetContent(this.kind.SaveToTableName);
        }
        /// <summary>
        /// 运行解析程序，获取每一页中的Url保存到数据库
        /// </summary>
        private void RunPages()
        {
            foreach (Rule rule in kind.RuleList)
            {
                ParseStepByStep(this.rootElements, rule);//从分类页面获取Url列表，保存到数据库
            }
        }
        /// <summary>
        /// 根据分类页面URL和初始步骤，获取第一层 initElements 元素
        /// </summary>
        /// <param name="sourceUrl">分类页面URL</param>
        /// <param name="initStep">初始步骤</param>
        /// <returns></returns>
        private Elements GetInitRootElements(string sourceUrl, Step initStep)
        {
            Elements elements = new Elements();
            try
            {
                doc = NSoup.NSoupClient.Connect(sourceUrl).Get();
            }
            catch (Exception ex)
            {
                pageStopCrawlCount++;
                LogHelper.Log.Error(" NSoup.NSoupClient.Connect(sourceUrl).Get() 失败。sourceUrl=" + sourceUrl, ex);
                return elements;
            }
            
            Element body = doc.Body;

            elements = ParseElements.GetElements(initStep, body);
            if (elements.Count == 0)
            {
                pageStopCrawlCount++;//没有结果的页面也加1
            }
            return elements;
        }
        /// <summary>
        /// 获取下一页Url
        /// </summary>
        /// <returns></returns>
        private string GetNextPageUrl() 
        {
            string nextPageUrl = null;            
            
            if (kind.PageRule.PageType == PageType.Number)
            {
                if (kind.PageRule.SpecialUrl != null && kind.PageRule.SpecialUrl.Count > 0)
                {

                }
            }
            else if (kind.PageRule.PageType == PageType.FixedNumber)
            {
            }
            else if (kind.PageRule.PageType == PageType.NextPageRule)
            {
                Rule r = kind.PageRule.NextPageRule;
                nextPageUrl = GetNextPageUrlString(this.rootElements, r);
            }
            return nextPageUrl;
        }
        /// <summary>
        /// 获取下一页Url地址
        /// </summary>
        /// <param name="initElements"></param>
        /// <param name="rule"></param>
        /// <returns></returns>
        private string GetNextPageUrlString(Elements initElements, Rule rule)
        {
            string nextPageUrl = null;
            //1、根据rule、step获取包含下一页链接的元素
            initElements = ParseElements.GetElementsByStep(initElements, rule);

            //2、根据Property获取下一页Url
            if (rule.NextProperty())
            {
                while (rule.CurrentProperty.NextStep())//属性中还有需要处理的步骤
                {
                    Step step = rule.CurrentProperty.CurrentStep;
                    initElements = ParseElements.GetAndMergeElements(initElements, step);
                    if (initElements.Count == 0)
                    {
                        break;
                    }
                }
                rule.CurrentProperty.ResetNextStep();
                //获取下一页Url
                nextPageUrl = ParseElements.GetResultStringByResult(initElements, rule.CurrentProperty.Result);
            }
            rule.ResetNextProperty();
            return nextPageUrl;
        }
        /// <summary>
        /// 按照Rule中的Step规则，解析出符合条件的元素列表。供Property获取最终结果用
        /// </summary>
        /// <param name="elements"></param>
        /// <param name="rule"></param>
        /// <returns></returns>
        //private Elements GetElementsByStep(Elements elements, Rule rule)
        //{
        //    //elements.Count > 0 判断 GetAndMergeElements 返回的initElements新结果集是否有数据，没有就跳出循环
        //    while ( rule.NextStep() && elements.Count > 0)
        //    {
        //        //按照step.GetBy规则，遍历elements，合并每个节点的结果。将新结果赋给elements，迭代下一个step操作。
        //        elements = ParseElements.GetAndMergeElements(elements, rule.CurrentStep);                
        //    }
        //    rule.ResetNextStep();
        //    return elements;
        //}
        /// <summary>
        /// 根据配置文件的step、leaf节点解析出详细页面url
        /// </summary>
        /// <param name="initElements"></param>
        /// <param name="rule"></param>
        private void ParseStepByStep(Elements initElements, Rule rule)
        {           
            // 当前页面中抓取到的Url数        
            int pageUrlCount = 0;
            // 当前页已经存在的Url数。如果加到和pageUrlCount相同，说明当前页已经抓取过。pageStopCrawlCount++        
            int pageSameUrlCount = 0;

            StringBuilder sb = new StringBuilder();

            //1、按照Rule中的Step规则，解析出符合条件的元素列表。供Property获取最终结果用
            initElements = ParseElements.GetElementsByStep(initElements, rule);          
            pageUrlCount = initElements.Count;//记录当前页面有多少个结果

            //2、解析出Property属性中的结果
            ResultKind resultKind = new ResultKind();
            resultKind.SiteKindId = this.kind.SiteKindId;
            resultKind.KindPageUrl = this.currentPageUrl;
            resultKind.SaveToTableName = this.kind.SaveToTableName;
            //将rule/step解析到的Elements保存到stepElements
            Elements stepElements;
            bool isTitleLengthOK = true;
            for (int i = 0; i < initElements.Count; i++ )
            {
                stepElements = new Elements();
                stepElements.Add(initElements[i]);//从列表中取一个元素放到stepElements中，然后解析出此元素中包含的每个结果属性

                while (rule.NextProperty())
                {
                    while (rule.CurrentProperty.NextStep())//属性中还有需要处理的步骤
                    {
                        Step step = rule.CurrentProperty.CurrentStep;
                        stepElements = ParseElements.GetAndMergeElements(stepElements, step);
                        if (stepElements.Count == 0)
                        {
                            break;
                        }
                    }
                    rule.CurrentProperty.ResetNextStep();                    

                    //获取最终属性结果
                    string result = ParseElements.GetResultStringByResult(stepElements, rule.CurrentProperty.Result);

                    //生成实体对象并赋值
                    isTitleLengthOK = SetResult(resultKind, rule.CurrentProperty.PropertyName, result);
                    if (!isTitleLengthOK)
                    {
                        break;//title长度不合法，跳出此结果，继续下一结果解析
                    }
                    //获取一个属性后,stepElements已经只包含一个结果属性，要重新给stepElements赋值为整个结果元素集合
                    stepElements = new Elements();
                    stepElements.Add(initElements[i]);
                }
                rule.ResetNextProperty();
                if (isTitleLengthOK)//如果title长度合法，保存结果到数据库
                {                    
                    bool existSameUrl = SaveResultToDB.SaveKindUrl(resultKind);
                    if (existSameUrl)//如果url已经存在
                    {
                        pageSameUrlCount++;
                    }
                    else
                    {
                        hasNewUrlList = true;
                    }
                }
                else//如果title长度不合法, 本页Url记录数减一
                {
                    pageUrlCount--;
                }               
            }
            if (pageSameUrlCount == pageUrlCount)
            {
                pageStopCrawlCount++;                
            }
            else//如果有新的Url存在，重置为0
            {
                pageStopCrawlCount = 0;               
            }
        }
        /// <summary>
        /// 设置结果对象值. title时判断最小长度是否满足xml配置。不满足返回false
        /// </summary>
        /// <param name="result"></param>
        /// <param name="propertyName"></param>
        /// <param name="resultStr"></param>
        private bool SetResult(ResultKind result, string propertyName, string resultStr)
        {
            bool isLengthOK = true;
            switch (propertyName.ToLower())
            {
                case "url":
                    result.Url = resultStr;
                    break;
                case "title":
                    result.Title = resultStr;
                    isLengthOK = resultStr.Length >= this.minLinkTextLength;//如果是标题，判断长度是否满足要求
                    break;
                case "img":
                    result.Img = resultStr;
                    break;
                case "summary":
                    result.Summary = resultStr;
                    break;
                case "sitekindid":
                    result.SiteKindId = resultStr;
                    break;
                case "author":
                    result.Author = resultStr;
                    break;
                case "actor":
                    result.Actor = resultStr;
                    break;
                case "year":
                    result.Year = resultStr;
                    break;
                case "area":
                    result.Area = resultStr;
                    break;
                default:
                    break;
            }
            return isLengthOK;
        }
        
        /// <summary>
        /// 遍历elements节点，获取每个节点中符合step条件的子节点集合；合并所有子节点集合，作为最终的results结果集
        /// </summary>
        /// <param name="elements"></param>
        /// <param name="step"></param>
        /// <returns></returns>
        //private Elements GetAndMergeElements(Elements elements, Step step)
        //{
        //    Elements results = new Elements();
            
        //    foreach (Element e in elements)
        //    {
        //        Elements eles = ParseElements.GetElements(step, e);
        //        foreach (Element r in eles)
        //        {
        //            results.Add(r);//将满足Step条件的元素添加到结果集                 
        //        }               
        //    }
        //    return results;
        //}
        //空标签
        private static readonly string[] emptyTags = {
            "meta", "link", "base", "frame", "img", "br", "wbr", "embed", "hr", "input", "keygen", "col", "command", "device"
        };        
        
        /// <summary>
        /// 获取满足step条件的元素集合
        /// </summary>
        /// <param name="step"></param>
        /// <param name="element"></param>        
        //private Elements GetElements(Step step, Element element)
        //{
        //    Elements results = new Elements();
        //    int index;
        //    switch (step.GetBy)
        //    {
        //        case GetBy.Id:
        //            Element ele = element.GetElementById(step.Name);
        //            if (ele != null && ele.ChildNodes.Count > 0)
        //            {
        //                results.Add(ele);
        //            }
        //            break;
        //        case GetBy.NextElementSibling:
        //            Element eleNext = element.NextElementSibling;
        //            if (eleNext != null)
        //            {
        //                results.Add(eleNext);
        //            }
        //            break;
        //        case GetBy.PreviousElementSibling:
        //            Element elePre = element.PreviousElementSibling;
        //            if (elePre != null)
        //            {
        //                results.Add(elePre);
        //            }
        //            break;
        //        case GetBy.Tag:
        //            results = element.GetElementsByTag(step.Name);
        //            break;
        //        case GetBy.Class:
        //            results = element.GetElementsByClass(step.Name);
        //            break;
        //        case GetBy.ContainText:
        //            results = element.GetElementsContainingText(step.Name);
        //            break;
        //        case GetBy.ContainOwnText:
        //            results = element.GetElementsContainingOwnText(step.Name);
        //            break;
        //        case GetBy.MatchText:
        //            results = element.GetElementsMatchingText(step.Name);
        //            break;
        //        case GetBy.MatchOwnText:
        //            results = element.GetElementsMatchingOwnText(step.Name);
        //            break;
        //        case GetBy.Attr:
        //            results = element.GetElementsByAttribute(step.Name);
        //            break;
        //        case GetBy.AttrStart:
        //            results = element.GetElementsByAttributeStarting(step.Name);
        //            break;
        //        case GetBy.IndexEquals:
        //            int.TryParse(step.Name, out index);
        //            results = element.GetElementsByIndexEquals(index);
        //            break;
        //        case GetBy.IndexLessThan:
        //            int.TryParse(step.Name, out index);
        //            results = element.GetElementsByIndexLessThan(index);
        //            break;
        //        case GetBy.IndexGreaterThan:
        //            int.TryParse(step.Name, out index);
        //            results = element.GetElementsByIndexGreaterThan(index);
        //            break;                
        //        //属性名，属性值
        //        case GetBy.AttrValue:
        //            results = element.GetElementsByAttributeValue(step.Name, step.Value);
        //            break;
        //        case GetBy.AttrValueStart:
        //            results = element.GetElementsByAttributeValueStarting(step.Name, step.Value);
        //            break;
        //        case GetBy.AttrValueEnd:
        //            results = element.GetElementsByAttributeValueEnding(step.Name, step.Value);
        //            break;
        //        case GetBy.AttrValueContain:
        //            results = element.GetElementsByAttributeValueContaining(step.Name, step.Value);
        //            break;
        //        case GetBy.AttrValueMatch:
        //            results = element.GetElementsByAttributeValueMatching(step.Name, step.Value);
        //            break;
        //        case GetBy.AttrValueContainDateNow:
        //            string date = DateTime.Now.ToString(step.Value);//通过step.Value指定的格式，格式化当前日期
        //            results = element.GetElementsByAttributeValueContaining(step.Name, date);
        //            break;

        //        case GetBy.RemoveTag:
        //            Elements tagElements = element.GetElementsByTag(step.Name);
        //            foreach (Element tag in tagElements)
        //            {
        //                tag.Remove();
        //            }
                   
        //            results.Add(element);                    
        //            break;
        //        default:
        //            LogHelper.Log.Error("GetElements(Step step, Element element) GetBy=" + step.GetBy + "不合法。");
        //            break;
        //    }
        //    //应用Step的过滤条件
        //    results = ParseElements.GetElementByFilterList(step.FilterList, results);          
        //    return results;
        //}
        
        /// <summary>
        /// 对所有Elements应用List&lt;Filter&gt;过滤条件
        /// </summary>
        /// <param name="filterList"></param>
        /// <param name="elements"></param>
        /// <returns></returns>
        //private Elements FilterStepElements(List<Filter> filterList, Elements elements)
        //{
        //    Elements results = new Elements();
        //    if (filterList != null && filterList.Count > 0)
        //    {
        //        foreach (Filter filter in filterList)//使用每个过滤条件
        //        {
        //            foreach (Element ele in elements)//过滤出符合条件的元素
        //            {
        //                Element e = ParseElements.GetElementByStepFilter(filter, ele);
        //                if (e != null)
        //                {
        //                    results.Add(e);
        //                }
        //            }

        //            if (results.Count == 0)
        //            {
        //                break;
        //            }
        //            else
        //            {
        //                elements = results;
        //            }
        //        }
        //    }
        //    else
        //    {
        //        return elements;
        //    }
        //    return results;
        //}
        /// <summary>
        /// 判断元素是否满足Step的Filter过滤条件。满足原样返回，不满足返回null
        /// </summary>
        /// <param name="Filter"></param>
        /// <param name="element"></param>
        /// <returns></returns>
        //private Element FilterStepElement(Filter filter, Element element)
        //{
        //    Element result = null;
        //    element.HasAttr("");
        //    switch (filter.GetBy)
        //    {
        //        case GetBy.StepFilterHasAttr:
        //            result = element.HasAttr(filter.Name) ? element : null;
        //            break;
        //        case GetBy.StepFilterAttrContains:
        //            if (element.HasAttr(filter.Name))
        //            {
        //                result = element.Attr(filter.Name).Contains(filter.Value) ? element : null;
        //            }
        //            break;
        //        case GetBy.StepFilterHasClass:
        //            result = element.HasClass(filter.Name) ? element : null;
        //            break;
        //        case GetBy.StepFilterOwnTextContains:
        //            result = element.OwnText().Contains(filter.Name) ? element : null;
        //            break;
        //        case GetBy.StepFilterTextContains:
        //            result = element.Text().Contains(filter.Name) ? element : null;
        //            break;
        //        case GetBy.StepFilterHtmlContains:
        //            result = element.Html().Contains(filter.Name) ? element : null;
        //            break;
        //        case GetBy.StepFilterValContains:
        //            result = element.Val().Contains(filter.Name) ? element : null;
        //            break;
        //        case GetBy.RemoveTag:
        //            Elements tagElements = element.GetElementsByTag(filter.Name);
        //            foreach (Element tag in tagElements)
        //            {
        //                tag.Remove();
        //            }
        //            result = element;
        //            break;

        //        default:
        //            LogHelper.Log.Error("FilterStepElement() GetBy=" + filter.GetBy + "不合法。");
        //            break;
        //    }
        //    return result;
        //}
        /// <summary>
        /// 获取最终结果字符串
        /// </summary>
        /// <param name="elements"></param>
        /// <param name="result"></param>
        /// <returns></returns>
        //private string GetResultString(Elements elements, XmlResult result)
        //{
        //    string str = null;
        //    if (elements == null || elements.Count < 1)
        //    {
        //        return str;
        //    }

        //    switch (result.GetBy)
        //    {
        //        case GetBy.ResultInAttr:
        //            str = elements[0].Attr(result.Name);
        //            break;
        //        case GetBy.ResultInText:
        //            str = elements[0].Text();
        //            break;
        //        case GetBy.ResultInHtml:
        //            str = elements[0].Html();
        //            break;
        //        default:
        //            break;
        //    }
        //    if (result.ResultFilter != null && result.ResultFilter.Count > 0)
        //    {
        //        str = ParseElements.GetResultStringByFilter(str, result.ResultFilter);
        //    }

        //    return StringHelper.RemoveMoreWhitespace(str);//删除字符串中多余的空格，只留1个         
        //}

        /// <summary>
        /// 从Tag的字符串[ 属性 / Text / Html ]中获取最终的结果字符串
        /// </summary>
        /// <param name="elements"></param>
        /// <param name="step"></param>
        /// <returns></returns>
        //private string FilterResultString(string tagString, List<Filter> filterList)
        //{            
        //    if (filterList == null || filterList.Count < 1)
        //    {
        //        return tagString;//直接返回
        //    }

        //    string result = tagString;
        //    foreach (Filter filter in filterList)
        //    {
        //        if (filter.GetBy == GetBy.ResultFilterBetween)//获取指定字符串之间的字符串，不包含字符串本身
        //        {
        //            int start = result.IndexOf(filter.Name);
        //            if (start != -1)
        //            {
        //                int end = result.IndexOf(filter.Name, start);
        //                if (end != -1)
        //                {
        //                    result = result.Substring(start + filter.Name.Length, end - start - filter.Name.Length);
        //                }
        //            }
        //        }
        //        else if (filter.GetBy == GetBy.ResultFilterReplace)//删除/替换
        //        {
        //            result = result.Replace(filter.Name, filter.Value);
        //        }
        //        else if (filter.GetBy == GetBy.ResultFilterMatch)//通过正则表达式获取结果字符串
        //        {
        //            Regex reg = new Regex(filter.Value, RegexOptions.IgnoreCase);
        //            Match match = reg.Match(result);
        //            if (match.Success)
        //            {
        //                result = match.Result("${result}");
        //            }
        //        }
        //    }

        //    return result;
        //}
    }
}
